import pandas as pd
from selenium import webdriver
import time
from lxml import etree
from selenium.webdriver.chrome.options import Options

# 关闭浏览器显示
chrome_options = Options()
chrome_options.add_argument("--headless")       # define headless
driver = webdriver.Chrome(chrome_options=chrome_options)

# 创建（用户，ID，链接，粉丝，简介）列表
id_list = []
href_list = []
fans_list = []
info_list = []
user1_list = []
weibo_list = []

for k in range(1, 200):
    url = 'https://d.weibo.com/1087030002_2975_1003_0?page='+str(k)
    driver.get(url)
    # 使用selenium获取页面，等待10秒，使页面加载完
    time.sleep(8)
    html = etree.HTML(driver.page_source)

    # 使用xpath对页面进行解析，提取出相应内容
    hrefs = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul//li/dl/dd[1]/div[1]/a[1]/@href')    # 用户主页地址
    users = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul//li/dl/dd[1]/div[1]/a[1]/@title')   # 用户名
    fans = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul//li/dl/dd[1]/div[2]/span[2]/em/text()')  # 粉丝数
    information = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul//li/dl/dd[1]/div[4]/span')    # 简介/微博认证
    user_id = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul//li/dl/dd[1]/div[1]/a[1]/@usercard')  # 用户ID
    weibos = html.xpath('//*[@id="Pl_Core_F4RightUserList__4"]/div/div/div/div/div[1]/ul/li/dl/dd[1]/div[2]/span[3]/em')
    # 将每一页的列表内容转到总列表中
    for Href in hrefs:
        href = 'https:' + Href
        href_list.append(href)
    for Fan in fans:
        fan = Fan.replace('万', '0000')
        fans_list.append(fan)
    for info in information:
        info_list.append(info.text)
    for user in users:
        user1_list.append(user)
    for ID in user_id:
        id = ID[3:13]
        id_list.append(id)
    for weibo in weibos:
        weibo_list.append(weibo.text)

    # 保存为CSV文件
    show_table = {
        '用户名': user1_list,
        '简介/微博认证': info_list,
        '粉丝数': fans_list,
        '用户ID': id_list,
        '微博主页地址': href_list,
        '发博数': weibo_list
    }
    show_table_df = pd.DataFrame(show_table)
    show_table_df.to_csv('user_star.csv', index=False, encoding='UTF-8')

    print("********第"+str(k)+"页用户爬取完成*********")
